# -*- coding:utf-8 -*-

from scrapy import Request
from scrapy.spiders import Spider
from scrapyExercise.items import ToutiaoItem
from selenium import webdriver

class ToutiaoSpider(Spider):
    name = "toutiao"

    def __init__(self):
        self.driver = webdriver.PhantomJS(executable_path="/Users/stevez/workspace/pycharmSpace/scrapyExercise/files/package/phantomjs-2.1.1-macosx/bin/phantomjs")
        # self.driver = webdriver.Chrome(executable_path="/Users/stevez/workspace/pycharmSpace/scrapyExercise/files/package/chromedriver/chromedriver")

    def start_requests(self):
        url = "https://www.toutiao.com/ch/news_hot/"
        yield Request(url)

    def parse(self, response, **kwargs):
        item = ToutiaoItem()
        list_selector = response.xpath('//div[@class="feed-list"]/div')
        for one_selector in list_selector:
            item['title'] = one_selector.xpath('.//div[@class="title-box"]/a/text()').extract_first()
            item['source'] = one_selector.xpath('.//a[@class="footer-bar-action source"]/text()').extract_first()
            item['comment'] = one_selector.xpath('.//a[@class="footer-bar-action source"][2]/text()').extract_first()
            yield item